package com.zxl.grocery.modular.crawler.processor;

import com.zxl.grocery.modular.crawler.base.AbstractPageProcessor;
import com.zxl.grocery.modular.crawler.base.Constant;
import com.zxl.grocery.modular.crawler.entity.CrawlerNews;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;

import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * info:开发者头条爬虫程序
 * Created by Lzx on 2017/7/12.
 */
@Component
public class DevelopersProcessor extends AbstractPageProcessor {

    public static final String list = "http://toutiao.io/prev/[0-9]{4}-[0-9]{2}-[0-9]{2}";

    public static final String item = "http://toutiao.io/posts/\\w+";

    public static final String url = "http://toutiao.io/j/\\w+";

    @Override
    public String[] initTargetUrl() {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        String tUrl = "http://toutiao.io/prev/" + sdf.format(new Date());
        String[] targetUrls = new String[]{tUrl};
        return targetUrls;
    }

    @Override
    public void process(Page page) {
        if (page.getUrl().regex(list).match()) {
            page.addTargetRequests(page.getHtml().xpath("//div[@class='post']").regex(item).all());
//            page.addTargetRequests(page.getHtml().links().regex(list).all());
        } else {
            String title = page.getHtml().xpath("//div[@class='content']/h3/a/text()").toString();
            String link = page.getHtml().regex(url).toString();
            if (StringUtils.isNotEmpty(title) && StringUtils.isNotEmpty(link)) {
                CrawlerNews news = new CrawlerNews();
                news.setTitle(title);
                news.setInfo(title);
                news.setLink(link);
                news.setTypeId(Constant.Type_KaiFaZhe);
                news.setSourcesId(Constant.Sources_KaiFaZhe);
                page.putField("news", news);
            }
        }
    }

}
